require 'net/http'  
require 'uri'
require 'iconv'  
require 'rexml/document'
require 'tidy'

module PageThief
    ISO = "ISO-8859-2"
    WIN = "WINDOWS-1250"
    UTF = "UTF-8"

    TIDY_OPTIONS = {
      :ascii_chars=>false,
      :output_xhtml=>true,
      :show_errors=>0,
      :show_warnings=>false,
      :wrap=>0,
      :wrap_sections=>false,
      :force_output=>true,
      :quiet=>true,
      :tidy_mark=>false
    } 

    def self.get_page_dom(url, encoding, do_clean_xml = false)
      Tidy.path = 'C:\Programs\ruby\lib\libtidy.dll'
      uri = URI.parse(url)
      http = Net::HTTP.new(uri.host, uri.port)
      path = uri.path.dup # required so we don't modify path
      path << "?#{uri.query}" if uri.query  
      headers = {}
      response = http.request_get(path, headers)
#      raise response.class.to_s
#      raise "Page does not exist" unless response.kind_of? Net::HTTPSuccess
      response.set_content_type("text/html; charset=" + encoding) 
      html = response.body 
#      raise html
      html = Iconv.iconv('utf-8//IGNORE', encoding, response.body)[0]
      xml = Tidy.open(TIDY_OPTIONS) do |tidy|
        tidy.options.input_encoding = 'utf8'
        tidy.options.output_encoding = 'utf8'  
        xml = tidy.clean html
        xml = self.clean_xml(xml) if do_clean_xml
        xml = self.clear_header_nons(xml)
#        raise xml
        rex = REXML::Document.new xml
        rex.root
      end
    end

    def self.clear_header_nons xml
      new_xml = '<html>'
      new_xml << xml[xml.index('<body')...xml.length]
      new_xml
    end  
    
    def self.clear_header xml
      new_xml = '<html xmlns="http://www.w3.org/1999/xhtml"><head>'
      new_xml += xml[xml.index('</head>')...xml.length]
      new_xml
    end
    
    # Tidy does not handle with <![... elems
    def self.clean_xml xml
      xml.gsub('<![','')
    end
end